#!/usr/bin/env bash
# deepdive-do -- Runs necessary processes to get something done
# > deepdive do TARGET...
##
set -euo pipefail

# record present working directory so processes can optionally recover it
export DEEPDIVE_PWD="$PWD"

. deepdive-do-getopts.sh
targets="$*"
. resolve-args-to-do.sh
cd "$DEEPDIVE_APP"

: ${DEEPDIVE_PLAN_EDIT:?}
: ${DEEPDIVE_LOG_LEVEL:?}

if deepdive-done "$@"; then
    if [[ $DEEPDIVE_LOG_LEVEL -gt 0 ]]; then
        cd run
        echo "All done at $(format_timestamp $(ls -t "$@" | head -1)):"
        echo " $targets"
        echo "To see more details, use: deepdive plan"
        echo "To redo them anyway, use: deepdive redo"
    fi
    exit 0
fi

# shorthands for setting -v flag dependeing on DEEPDIVE_LOG_LEVEL
verboseLevel1=; [[ $DEEPDIVE_LOG_LEVEL -lt 1 ]] || verboseLevel1=v
verboseLevel2=; [[ $DEEPDIVE_LOG_LEVEL -lt 2 ]] || verboseLevel2=v

# create a directory for running
runDir=$(date +%Y%m%d/%H%M%S.%N)
mkdir -p run/"$runDir"

# forward signals to descendants to make sure no dangling processes remain
signals="HUP INT QUIT TERM"
signal_pg() {
    local sig=$1
    # send given signal to the process group
    exec ps_descendants $$ | xargs -t kill -$sig || true
}
keep_signal_pg() {
    local sig=$1
    # make sure all processes in this process group, hence its descendent are terminated
    # (sometimes UDF processes don't terminate upon signals sent from tty)
    echo "Sending SIG$sig to descendants of PID $$"
    trap 'keep_signal_pg KILL' $signals  # or send KILL if it receives signal once again
    signal_pg $sig
    case $sig in KILL) return; esac  # just KILLed everyone and end of story
    # until no process remains in the process group
    # keep sending the same signal with increasing interval
    local timeout=1
    while [[ -n $(exec ps_descendants $$) ]]; do
        echo "Processes still alive, sending SIG$sig again to descendants of PID $$ in $timeout secs"
        sleep $timeout && signal_pg $sig && let timeout*=2 || {
            # or send KILL if something goes wrong
            keep_signal_pg KILL
            return 1
        }
    done
}
for sig in $signals
do trap "keep_signal_pg TERM" $sig  # XXX sending TERM instead of $sig as some children don't respond to other ones
done

# prepare the execution plan for given targets
cd "$DEEPDIVE_APP"/run
{
    # some metadata
    echo "#!/usr/bin/env bash"
    echo "set -veu"
    echo "################################################################################"
    echo "# Host: $HOSTNAME"
    echo "# DeepDive: $(deepdive-version 2>/dev/null | head -1)"
    echo "export PATH=$(escape4sh "$(dirname "$DEEPDIVE_SHELL")"):\"\$PATH\""
    echo "export DEEPDIVE_PWD=$(escape4sh "$DEEPDIVE_PWD")"
    echo "export DEEPDIVE_APP=$(escape4sh "$DEEPDIVE_APP")"
    echo 'cd "$DEEPDIVE_APP"/run'
    echo "export DEEPDIVE_RUN_ID=$(escape4sh "$runDir")"
    echo "# Plan: $runDir/plan.sh"
    echo "# Targets: $targets"
    echo "################################################################################"
    # and the plan
    deepdive-plan "$@" | tail -n +2
} |
tee "$runDir"/plan.orig.sh >"$runDir"/plan.sh
# record DeepDive version
deepdive version >"$runDir"/deepdive.version
# record environment
env >"$runDir"/environ

# provide a chance to edit plan in a tty unless told not to ask
if [[ -t 0 && -t 1 && $VISUAL != true && $VISUAL != : ]] && $DEEPDIVE_PLAN_EDIT; then
    $VISUAL "$runDir"/plan.sh || true  # ignore error from user's visual editor and rely on timestamp
    [[ "$runDir"/plan.sh -nt "$runDir"/plan.orig.sh ]] || {
        rm -rf "$runDir"
        error "Canceled execution" || exit 130
    }
fi

# remove original unless modified
! diff -q "$runDir"/plan{,.orig}.sh || rm -f "$runDir"/plan.orig.sh

# turn it into an executable
chmod +x "$runDir"/plan.sh

# maintain a few convenience symlinks
#  making sure we clean up upon exit
cleanup() {
    cd "$DEEPDIVE_APP"
    [[ ! run/RUNNING -ef run/"$runDir" ]] || rm -f run/RUNNING
    # make sure no descendant processes are left behind
    [[ -z $(exec ps_descendants $$) ]] || keep_signal_pg TERM
}
trap cleanup EXIT
#  and leaving an ABORTED symlink upon error
abort() {
    cd "$DEEPDIVE_APP"
    [[ $DEEPDIVE_LOG_LEVEL -gt 1 ]] || error-from-file run/"$runDir"/run.log || true
    [[ ! -e run/"$runDir" ]] || ln -sfnv "$runDir" run/ABORTED >&2
}
trap abort ERR
cd "$DEEPDIVE_APP"
ln -sfn$verboseLevel1 "$runDir" run/RUNNING
ln -sfn$verboseLevel2 "$runDir" run/LATEST

# run it
( cd "$DEEPDIVE_APP"/run

# NOTE the exec redirections to process substitution below are applied in reverse order
# i.e., the first one gets the output already processed by the ones that follow it,
# allowing us to produce less verbose output and do separate logging.

# reduce the noise unless the log level is verbose enough
case $DEEPDIVE_LOG_LEVEL in
    0) # display nothing
        exec >/dev/null
        ;;
    1) # display only the important lines in the plan
        # NOTE the pattern below relies on at least two hashes (`##`) present
        # before the `last done:` marker, to be produced by deepdive-plan
        exec > >(sed '/ ###* last done: /!d; /^..........................     # /d; s/ ###* .*//')
        ;;
    *) # show everything that's logged
esac

# keep a unified log file with timestamped stdout and stderr
exec > >(tee "$runDir"/run.log) 2>&1
ln -sfn run.log "$runDir"/log.txt # XXX for backward compatibility with pre-0.7.x, keep a symlink

# also keep a separate log file for stdout and stderr with timestamps on each line
exec 1> >(logging-with-ts "$runDir"/run.out >&1)
exec 2> >(logging-with-ts "$runDir"/run.err >&2)

export DEEPDIVE_ERROR_PREFIX='[ERROR] '  # prefixing error messages in logs
# XXX some legacy variables
export APP_HOME="$DEEPDIVE_APP"                     # XXX legacy
export DEEPDIVE_OUTPUT="$DEEPDIVE_APP/run/$runDir"  # XXX legacy
# NOTE more environment variables should be added to top of the plan not here

exec "$runDir"/plan.sh
)

# leave a symlink to the latest successful run
cd "$DEEPDIVE_APP"
[[ ! -e run/FINISHED ]] || mv -f$verboseLevel2 --no-target-directory run/FINISHED run/FINISHED~
ln -sfnv "$runDir" run/FINISHED
